# coding=utf-8

import pdb
import analysisPage
import utils
import random
import json

if __name__ == '__main__':
    
     # 自定义header头信息
    headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36'}
    print('开始执行脚本=====>')
    i = 0
    try:
        file_handle = open('trainingData.log', encoding="utf-8")
        lines = file_handle.readlines()
        for line in lines:
            if line:
                i += 1
                line_arr = line.split(',')
                url = line_arr[0].strip()
                method = line_arr[1].strip()
                proxies = analysisPage.getRedisProxies()
                analysisPage.getListLinks(url, headers, proxies, method)    
        print(f'脚本执行结束，总处理 {i}条数据=====>')
    finally:
        if file_handle:
            file_handle.close()
   
    # excel_url = 'http://www.ccgp-shaanxi.gov.cn/notice/list.do?noticetype=3&province=province'
    # excel_url = 'http://www.qhggzyjy.gov.cn/ggzy/jyxx/001001/001001001/secondPage.html'

    """
    <<<<<<<<<<<<<<处理不了的start>>>>>>>>>>>>>>>>>>>
    """
    #iframe内容抓取不到，需再次请求iframe的接口获取内容
    # excel_url = 'http://ggzy.huaian.gov.cn/EpointWeb/showinfo/jyxxsearch.aspx'

    #页面结构规则异常
    # excel_url = 'http://www.lntb.gov.cn/#/listMore?index=0'

    #数据列表锚点跳转
    # excel_url = 'https://b2b.10086.cn/b2b/main/listVendorNotice.html?noticeType=2'

    """
    <<<<<<<<<<<<<<处理不了的start>>>>>>>>>>>>>>>>>>>
    """
    # excel_url = 'http://ggzyjy.baiyin.gov.cn/InfoPage/TradeInfomation.aspx?state=1,2,3'
    #excel_url = 'http://www.ccgp-shaanxi.gov.cn/notice/list.do?noticetype=3&province=province'
    # excel_url = 'http://www.qhggzyjy.gov.cn/huangn/jyxx/tradeInfo.html'
    #excel_url = 'http://ggzy.huaian.gov.cn/EpointWeb/showinfo/jyxxsearch.aspx'
    #excel_url = 'http://www.cau.edu.cn/col/col10353/index.html'
    # excel_url = 'http://www.ccgp-qinghai.gov.cn/'
    #excel_url = 'http://ggzyjy.abazhou.gov.cn/ggzyjyzx/c106701/common_list_2.shtml'
    #proxies = {}
    # 根据详情页链接的正则匹配方案
    #analysisPage.getDetailList(excel_url,headers,proxies)
    # 根据列表页面详情链接相识度选出相识度高的为详情链接
    #analysisPage.getListLinks(excel_url, headers, proxies)


    # 详情页内容分析

    #detail_url = 'https://b2b.10086.cn/b2b/main/viewNoticeContent.html?noticeBean.id=764747'
    #target_title = '大数据机房项目（第二批）_比选公告'

    #analysisPage.analysisPageInfo(detail_url, headers, proxies, target_title)